package com.spark;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Created by TRS on 2017/6/9.
 */
public class ZJOL {

    public static String PATH = "hdfs://10.200.73.90:9000/zjol/";

    public static OkHttpClient client = new OkHttpClient();

    public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1);


    public static void main(String[] args) throws IOException {

        getNews(21550000, 10000);

    }


    public static void getNews(int id, final int size) {


        final List<String> lines = new ArrayList<String>();

        while (id > 0) {
            final Integer nId = id;
            fixedThreadPool.submit(new Runnable() {
                public void run() {
                    try {
                        int end = nId - size;
                        List<String> news = new ArrayList<String>();
                        for (int subId = nId; subId > end; subId--) {
                            String newId = "0" + subId;
                            String json = getJson(newId);

                            if (!json.equals("nocontent")) {
                                try {
                                    JSONObject okJosn = JSON.parseObject(json);
                                    news.add(okJosn.toJSONString());
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }

                            }
                            System.out.println(news.size());
                        }

                        String fileNmae = PATH + nId + ".json";
                        HDFSUtill.writeLine(fileNmae, news);
                        System.out.println(fileNmae + " " + news.size());

                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
            id -= size;

        }
    }

    public static String getJson(String newID) {
        String newUrl = "http://m.zjol.com.cn/mzjol/api?action=getnews&newsid=" + newID;
        String json = null;
        Request request = new Request.Builder()
                .url(newUrl)
                .get()
                .build();
        Response response = null;
        try {
            response = client.newCall(request).execute();
            json = response.body().string();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return json;
    }

    public static List<String> getUrls(String url) {
        try {
            Document doc = Jsoup.connect(url).get();
            Elements links = doc.select("a[href]");
            for (Element e : links) {

                System.out.println(e.attr("abs:href"));
            }
        } catch (Exception e) {

        }
        return null;

    }

}
